Data is from August, 2017 sequencing. This is week 4 of experiment 1

Download and map:

Working on Whitney for read mapping. Directory: /Network/Servers/avalanche.plb.ucdavis.edu/Volumes/Mammoth/Users/jmaloof/2017/Wyoming-microbiome/20170830-data/

20170830 reads

Build index (not done…already exists)

cd ~/Sequences/ref_genomes/B_rapa/genome/V3.0

wget http://brassicadb.org/brad/datasets/pub/BrassicaceaeGenome/Brassica_rapa/V3.0/Brapa_genome_v3.0_cds.fasta.gz 

kallisto index -i B_rapa_CDS_V3.0_k31_kallisto_index Brapa_genome_v3.0_cds.fasta.gz

cd ~/2017/Wyoming-microbiome//20170830-samples/20170830-data

Map reads

mkdir kallisto_out_V3.0

#actually a fish loop
for file in (ls raw-fastq/2017-08-27/*.fastq.gz)
  echo $file
  set newfile (basename $file _R1_001.fastq.gz)
  kallisto quant -i ~/Sequences/ref_genomes/B_rapa/genome/V3.0/B_rapa_CDS_V3.0_k31_kallisto_index  -o kallisto_out_V3.0/$newfile --single -l 200 -s 40 -t 4 --plaintext $file
end

Move the counts to my local computer

NEED TO MODIFY BELOW

cd /Users/jmaloof/git/Br_Microbe_Paper_2021/RNA/input/20170830-samples
lftp sftp://whitney.plb.ucdavis.edu
  cd 2017/Wyoming-microbiome/20170830-samples/20170830-data
  mirror kallisto_out_V3.0

remove unused files

cd /Users/jmaloof/git/Br_Microbe_Paper_2021/RNA/input/20170830-samples/kallisto_out_V3.0
rm */*.json

compress tsv files

cd /Users/jmaloof/git/Br_Microbe_Paper_2021/RNA/input/20170830-samples/kallisto_out_V3.0
gzip */abundance.tsv

Get counts into R

library(tidyverse)
Registered S3 methods overwritten by 'dbplyr':
  method         from
  print.tbl_lazy     
  print.tbl_sql      
── Attaching packages ──────────────────────────────── tidyverse 1.3.1 ──
✓ ggplot2 3.3.3     ✓ purrr   0.3.4
✓ tibble  3.1.2     ✓ dplyr   1.0.6
✓ tidyr   1.1.3     ✓ stringr 1.4.0
✓ readr   1.4.0     ✓ forcats 0.5.1
── Conflicts ─────────────────────────────────── tidyverse_conflicts() ──
x dplyr::filter() masks stats::filter()
x dplyr::lag()    masks stats::lag()
library(stringr)
library(edgeR)
Loading required package: limma
head(kallisto_names)
[1] "wyo_leaf_R500_10_417_S65_L001" "wyo_leaf_R500_10_417_S65_L002"
[3] "wyo_leaf_R500_10_417_S65_L003" "wyo_leaf_R500_10_417_S65_L004"
[5] "wyo_leaf_R500_10_419_S66_L001" "wyo_leaf_R500_10_419_S66_L002"
counts <- tibble(sample = kallisto_names, file = kallisto_files) %>%
  mutate(countdata = map(kallisto_files, read_tsv))

── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)


── Column specification ─────────────────────────────────────────────────
cols(
  target_id = col_character(),
  length = col_double(),
  eff_length = col_double(),
  est_counts = col_double(),
  tpm = col_double()
)
head(counts)

reformat into rows = genes and columns = samples

counts <- counts %>% unnest(countdata) %>%
  mutate(sample = str_remove(sample, "_L.*")) %>% 
  select(sample, target_id, est_counts) %>%
  group_by(sample, target_id) %>%
  summarize(est_counts=sum(est_counts)) %>% # sum up counts from multiple lanes
  ungroup() %>%
  pivot_wider(id_cols = target_id,
              names_from = sample,
              values_from = est_counts)
`summarise()` has grouped output by 'sample'. You can override using the `.groups` argument.
head(counts)
dim(counts)
[1] 46250    25
write_csv(counts,"../../output/20170830_V3.0_raw_counts_.csv.gz")

make sample description data frame

#CONTINUE FROM HERE

sample.description <- tibble(sample=colnames(counts)[-1]) %>%
  separate(sample,
           c("location","tissue","genotype","block","pot"),
           remove=FALSE,
           convert=TRUE) 
head(sample.description)

## get additional metadata
sample.info <- readxl::read_excel("../input/wy001_RNAseq.xlsx",sheet = 1)
head(sample.info)

##combine
sample.description <- left_join(sample.description, sample.info)
sample.description <- sample.description %>% 
  mutate(group=paste(tissue,genotype,soil,autoclave,sep="_"))
head(sample.description)
sample.description %>% summarize(n_distinct(group))

summarize counts

pl.orig <- counts[,-1] %>% colSums() %>% tibble(sample=names(.),count=.) %>%
  ggplot(aes(x=sample,y=count)) + 
  geom_col() +
  theme(axis.text.x  = element_text(angle=90, vjust=0.5,size = 7)) 
pl.orig

load to edgeR, normalize

#confirm that everthing is in the right order
all(colnames(counts)[-1]==sample.description$sample)
dge <- DGEList(counts[,-1],
               group=sample.description$group,
               samples=sample.description,
               genes=counts$target_id)
dge <- calcNormFactors(dge)
barplot(dge$samples$lib.size)
ggplot(dge$samples,aes(x=sample,y=norm.factors,fill=tissue)) + geom_col() + 
  theme(axis.text.x  = element_text(angle=90, vjust=0.5,size = 7)) 
ggplot(dge$samples,aes(x=sample,y=norm.factors,fill=genotype)) + geom_col() + 
  theme(axis.text.x  = element_text(angle=90, vjust=0.5,size = 7)) 
ggplot(dge$samples,aes(x=sample,y=norm.factors,fill=as.factor(block))) + geom_col() +
  theme(axis.text.x  = element_text(angle=90, vjust=0.5,size = 7)) 

Looks like we should normalize separately for root and leaf

do separately for leaf and root values

counts.leaf <- counts %>% select(target_id, contains("leaf"))
counts.root <- counts %>% select(target_id, contains("root"))
sample.description.leaf <- sample.description %>% filter(tissue=="leaf")
sample.description.root <- sample.description %>% filter(tissue=="root")

Leaf

#confirm that everthing is in the right order
all(colnames(counts.leaf)[-1]==sample.description.leaf$sample)
dge.leaf <- DGEList(counts.leaf[,-1],
                    group=sample.description.leaf$group,
                    samples=sample.description.leaf,
                    genes=counts.leaf$target_id)
dge.leaf <- calcNormFactors(dge.leaf)

Root

#confirm that everthing is in the right order
all(colnames(counts.root)[-1]==sample.description.root$sample)
dge.root <- DGEList(counts.root[,-1],
                    group=sample.description.root$group,
                    samples=sample.description.root,
                    genes=counts.root$target_id)
dge.root <- calcNormFactors(dge.root)
save(dge.leaf,dge.root,sample.description.leaf,sample.description.root,file="../output/edgeR_dge_objects.Rdata")

Write out cpm values

cpm.leaf.w <- bind_cols(dge.leaf$gene,as_tibble(cpm(dge.leaf))) %>% as_tibble() %>% rename(transcript_ID=genes)
head(cpm.leaf.w)
write_csv(cpm.leaf.w,"../output/cpm_wide_20170617_leaf_samples.csv.gz")
cpm.root.w <- bind_cols(dge.root$gene,as_tibble(cpm(dge.root))) %>% as_tibble() %>% rename(transcript_ID=genes)
head(cpm.root.w)
write_csv(cpm.root.w,"../output/cpm_wide_20170617_root_samples.csv.gz")

Also let’s reformat this to long format and add metadata

cpm.leaf.long <- cpm.leaf.w %>% 
  gather(-transcript_ID,key = sample,value=cpm) %>%
  left_join(sample.description.leaf)
head(cpm.leaf.long)
write_csv(cpm.leaf.long,"../output/cpm_long_with_metadata_20170617_leaf_samples.csv.gz")
cpm.root.long <- cpm.root.w %>% 
  gather(-transcript_ID,key = sample,value=cpm) %>%
  left_join(sample.description.root)
head(cpm.root.long)
write_csv(cpm.root.long,"../output/cpm_long_with_metadata_20170617_root_samples.csv.gz")

compute and write out voom-transformed values

design.leaf <- model.matrix(~ sample.description.leaf$group)
dge4voom.leaf <- dge.leaf[rowSums(cpm(dge.leaf)>1) >= 6,,keep.lib.sizes=FALSE]
dge4voom.leaf <- calcNormFactors(dge4voom.leaf)
data.voom.leaf <- voom(dge4voom.leaf,design = design.leaf)
data.voom.exp.leaf <- bind_cols(data.voom.leaf$genes,as_tibble(data.voom.leaf$E)) %>%
  rename(transcript_ID=genes) %>% as_tibble()
head(data.voom.exp.leaf)
write_csv(data.voom.exp.leaf, "../output/voom_expression_20170617_T6_leaf_samples.csv.gz")
voom.long.leaf <- data.voom.exp.leaf %>% 
  gather(-transcript_ID,key = sample,value=expression) %>%
  left_join(sample.description.leaf)
head(voom.long.leaf)
hist(voom.long.leaf$expression)
write_csv(voom.long.leaf,"../output/voom_long_with_metadata_20170617_T6_leaf_samples.csv.gz")
design.root <- model.matrix(~ sample.description.root$group)
dge4voom.root <- dge.root[rowSums(cpm(dge.root)>1) >= 6,,keep.lib.sizes=FALSE]
dge4voom.root <- calcNormFactors(dge4voom.root)
data.voom.root <- voom(dge4voom.root,design = design.root)
data.voom.exp.root <- bind_cols(data.voom.root$genes,as_tibble(data.voom.root$E)) %>%
  rename(transcript_ID=genes) %>% as_tibble()
head(data.voom.exp.root)
write_csv(data.voom.exp.root, "../output/voom_expression_20170617_T6_root_samples.csv.gz")
voom.long.root <- data.voom.exp.root %>% 
  gather(-transcript_ID,key = sample,value=expression) %>%
  left_join(sample.description.root)
head(voom.long.root)
hist(voom.long.root$expression)
write_csv(voom.long.root,"../output/voom_long_with_metadata_20170617_T6_root_samples.csv.gz")

write it to irods

Need to run this yourself in terminal

iinit
icd /iplant/home/shared/ucd.brassica/analyses/Brapa_Microbiome_RNAseq/
for f in (ls cpm*)
    echo $f
    iput -vf $f
end
for f in (ls voom*)
    echo $f
    iput -vf $f
end

read distribution

Mike asked if the difference in normalization factors in leafs vs roots was due to high abundance of photosynthesis transcripts in leafs. (Although leafs have lower normalization factor)

Start by doing a simplistic normalization just by library size. Then look at distribution of most abundant counts.

counts.leaf.norm <- counts.leaf %>% 
  mutate_at(-1,funs(./sum(.))) %>% 
  transmute(target_id=target_id,mean= {select(.,-target_id) %>% rowMeans()}) %>%
  arrange(desc(mean)) %>% mutate(cumsum=cumsum(mean),rank=row_number(),sample="leaf")
counts.leaf.norm
counts.root.norm <- counts.root %>% 
  mutate_at(-1,funs(./sum(.))) %>% 
  transmute(target_id=target_id,mean= {select(.,-target_id) %>% rowMeans()}) %>%
  arrange(desc(mean)) %>% mutate(cumsum=cumsum(mean),rank=row_number(),sample="root")
counts.root.norm
rbind(counts.leaf.norm,counts.root.norm) %>%
  ggplot(aes(x=rank,y=cumsum,color=sample)) +
  geom_line() +xlim(0,20000)
rbind(counts.leaf.norm,counts.root.norm) %>%
  ggplot(aes(x=rank,y=mean,color=sample)) +
  geom_line()  + scale_y_log10()
rbind(counts.leaf.norm,counts.root.norm) %>% filter(rank < 41) %>%
  ggplot(aes(x=rank,y=mean,fill=sample)) +
  geom_col(position = "dodge") 
annotation <- read_csv("../../../Annotation/output/v3.0annotation/Brapa_V3.0_annotated.csv")
top.expressed.leaf <- counts.leaf.norm %>% filter(rank<21) %>% left_join(annotation,by=c("target_id"="name")) %>% select("target_id","mean","rank","AGI","At_symbol","At_description") %>% arrange(rank)
top.expressed.leaf
write_csv(top.expressed.leaf,"../output/top.expressed.leaf.csv")
top.expressed.root <- counts.root.norm %>% filter(rank<21) %>% left_join(annotation,by=c("target_id"="name")) %>% select("target_id","mean","rank","AGI","At_symbol","At_description") %>% arrange(rank)
top.expressed.root
write_csv(top.expressed.root,"../output/top.expressed.root.csv")
LS0tCnRpdGxlOiAiUk5BIGV4cHJlc3Npb24gYW5hbHlzaXMgb2YgQnJhc3NpY2EgTWljcm9iZSBEYXRhLiBJOiBwcmVwIGRhdGEiCm91dHB1dDogaHRtbF9ub3RlYm9vawotLS0KCkRhdGEgaXMgZnJvbSBBdWd1c3QsIDIwMTcgc2VxdWVuY2luZy4gIFRoaXMgaXMgd2VlayA0IG9mIGV4cGVyaW1lbnQgMQoKIyMgRG93bmxvYWQgYW5kIG1hcDoKCldvcmtpbmcgb24gV2hpdG5leSBmb3IgcmVhZCBtYXBwaW5nLiAgRGlyZWN0b3J5OiBgL05ldHdvcmsvU2VydmVycy9hdmFsYW5jaGUucGxiLnVjZGF2aXMuZWR1L1ZvbHVtZXMvTWFtbW90aC9Vc2Vycy9qbWFsb29mLzIwMTcvV3lvbWluZy1taWNyb2Jpb21lLzIwMTcwODMwLWRhdGEvYAoKIyMjIDIwMTcwODMwIHJlYWRzCgpCdWlsZCBpbmRleCAobm90IGRvbmUuLi5hbHJlYWR5IGV4aXN0cykKYGBge3IsIGVuZ2luZT0nYmFzaCcsIGV2YWw9RkFMU0V9CmNkIH4vU2VxdWVuY2VzL3JlZl9nZW5vbWVzL0JfcmFwYS9nZW5vbWUvVjMuMAoKd2dldCBodHRwOi8vYnJhc3NpY2FkYi5vcmcvYnJhZC9kYXRhc2V0cy9wdWIvQnJhc3NpY2FjZWFlR2Vub21lL0JyYXNzaWNhX3JhcGEvVjMuMC9CcmFwYV9nZW5vbWVfdjMuMF9jZHMuZmFzdGEuZ3ogCgprYWxsaXN0byBpbmRleCAtaSBCX3JhcGFfQ0RTX1YzLjBfazMxX2thbGxpc3RvX2luZGV4IEJyYXBhX2dlbm9tZV92My4wX2Nkcy5mYXN0YS5negoKY2Qgfi8yMDE3L1d5b21pbmctbWljcm9iaW9tZS8vMjAxNzA4MzAtc2FtcGxlcy8yMDE3MDgzMC1kYXRhCmBgYAoKTWFwIHJlYWRzCgpgYGB7ciwgZW5naW5lPSdiYXNoJyxldmFsPUZBTFNFfQpta2RpciBrYWxsaXN0b19vdXRfVjMuMAoKI2FjdHVhbGx5IGEgZmlzaCBsb29wCmZvciBmaWxlIGluIChscyByYXctZmFzdHEvMjAxNy0wOC0yNy8qLmZhc3RxLmd6KQogIGVjaG8gJGZpbGUKICBzZXQgbmV3ZmlsZSAoYmFzZW5hbWUgJGZpbGUgX1IxXzAwMS5mYXN0cS5neikKICBrYWxsaXN0byBxdWFudCAtaSB+L1NlcXVlbmNlcy9yZWZfZ2Vub21lcy9CX3JhcGEvZ2Vub21lL1YzLjAvQl9yYXBhX0NEU19WMy4wX2szMV9rYWxsaXN0b19pbmRleCAgLW8ga2FsbGlzdG9fb3V0X1YzLjAvJG5ld2ZpbGUgLS1zaW5nbGUgLWwgMjAwIC1zIDQwIC10IDQgLS1wbGFpbnRleHQgJGZpbGUKZW5kCmBgYAoKTW92ZSB0aGUgY291bnRzIHRvIG15IGxvY2FsIGNvbXB1dGVyCgojIyMgTkVFRCBUTyBNT0RJRlkgQkVMT1cKCmBgYHtyLCBlbmdpbmU9J2Jhc2gnLCBldmFsPUZBTFNFfQpjZCAvVXNlcnMvam1hbG9vZi9naXQvQnJfTWljcm9iZV9QYXBlcl8yMDIxL1JOQS9pbnB1dC8yMDE3MDgzMC1zYW1wbGVzCmxmdHAgc2Z0cDovL3doaXRuZXkucGxiLnVjZGF2aXMuZWR1CiAgY2QgMjAxNy9XeW9taW5nLW1pY3JvYmlvbWUvMjAxNzA4MzAtc2FtcGxlcy8yMDE3MDgzMC1kYXRhCiAgbWlycm9yIGthbGxpc3RvX291dF9WMy4wCmBgYAoKcmVtb3ZlIHVudXNlZCBmaWxlcwpgYGB7ciwgZW5naW5lPSdiYXNoJywgZXZhbD1GQUxTRX0KY2QgL1VzZXJzL2ptYWxvb2YvZ2l0L0JyX01pY3JvYmVfUGFwZXJfMjAyMS9STkEvaW5wdXQvMjAxNzA4MzAtc2FtcGxlcy9rYWxsaXN0b19vdXRfVjMuMApybSAqLyouanNvbgpgYGAKCmNvbXByZXNzIHRzdiBmaWxlcwpgYGB7ciwgZW5naW5lPSdiYXNoJywgZXZhbD1GQUxTRX0KY2QgL1VzZXJzL2ptYWxvb2YvZ2l0L0JyX01pY3JvYmVfUGFwZXJfMjAyMS9STkEvaW5wdXQvMjAxNzA4MzAtc2FtcGxlcy9rYWxsaXN0b19vdXRfVjMuMApnemlwICovYWJ1bmRhbmNlLnRzdgpgYGAKCgojIyBHZXQgY291bnRzIGludG8gUgoKYGBge3J9CmxpYnJhcnkodGlkeXZlcnNlKQpsaWJyYXJ5KHN0cmluZ3IpCmxpYnJhcnkoZWRnZVIpCmBgYAoKCgpgYGB7cn0Ka2FsbGlzdG9fZmlsZXMgPC0gZGlyKHBhdGggPSAiLi4vLi4vaW5wdXQvMjAxNzA4MzAtc2FtcGxlcy9rYWxsaXN0b19vdXRfVjMuMCIscGF0dGVybj0iYWJ1bmRhbmNlLnRzdiIscmVjdXJzaXZlID0gVFJVRSxmdWxsLm5hbWVzID0gVFJVRSkKa2FsbGlzdG9fbmFtZXMgPC0gc3RyX3NwbGl0KGthbGxpc3RvX2ZpbGVzLCIvIixzaW1wbGlmeT1UUlVFKVssNl0KaGVhZChrYWxsaXN0b19uYW1lcykKYGBgCgpgYGB7cn0KY291bnRzIDwtIHRpYmJsZShzYW1wbGUgPSBrYWxsaXN0b19uYW1lcywgZmlsZSA9IGthbGxpc3RvX2ZpbGVzKSAlPiUKICBtdXRhdGUoY291bnRkYXRhID0gbWFwKGthbGxpc3RvX2ZpbGVzLCByZWFkX3RzdikpICU+JQogIHNlbGVjdCgtZmlsZSkKCmhlYWQoY291bnRzKQpgYGAKCnJlZm9ybWF0IGludG8gcm93cyA9IGdlbmVzIGFuZCBjb2x1bW5zID0gc2FtcGxlcwpgYGB7cn0KY291bnRzIDwtIGNvdW50cyAlPiUgdW5uZXN0KGNvdW50ZGF0YSkgJT4lCiAgbXV0YXRlKHNhbXBsZSA9IHN0cl9yZW1vdmUoc2FtcGxlLCAiX0wuKiIpKSAlPiUgCiAgc2VsZWN0KHNhbXBsZSwgdGFyZ2V0X2lkLCBlc3RfY291bnRzKSAlPiUKICBncm91cF9ieShzYW1wbGUsIHRhcmdldF9pZCkgJT4lCiAgc3VtbWFyaXplKGVzdF9jb3VudHM9c3VtKGVzdF9jb3VudHMpKSAlPiUgIyBzdW0gdXAgY291bnRzIGZyb20gbXVsdGlwbGUgbGFuZXMKICB1bmdyb3VwKCkgJT4lCiAgcGl2b3Rfd2lkZXIoaWRfY29scyA9IHRhcmdldF9pZCwKICAgICAgICAgICAgICBuYW1lc19mcm9tID0gc2FtcGxlLAogICAgICAgICAgICAgIHZhbHVlc19mcm9tID0gZXN0X2NvdW50cykKYGBgCgpgYGB7cn0KaGVhZChjb3VudHMpCmRpbShjb3VudHMpCmBgYAoKYGBge3J9CndyaXRlX2Nzdihjb3VudHMsIi4uLy4uL291dHB1dC8yMDE3MDgzMF9WMy4wX3Jhd19jb3VudHNfLmNzdi5neiIpCmBgYAoKIyMgbWFrZSBzYW1wbGUgZGVzY3JpcHRpb24gZGF0YSBmcmFtZQoKI0NPTlRJTlVFIEZST00gSEVSRQoKYGBge3J9CnNhbXBsZS5kZXNjcmlwdGlvbiA8LSB0aWJibGUoc2FtcGxlPWNvbG5hbWVzKGNvdW50cylbLTFdKSAlPiUKICBzZXBhcmF0ZShzYW1wbGUsCiAgICAgICAgICAgYygibG9jYXRpb24iLCJ0aXNzdWUiLCJnZW5vdHlwZSIsImJsb2NrIiwicG90IiksCiAgICAgICAgICAgcmVtb3ZlPUZBTFNFLAogICAgICAgICAgIGNvbnZlcnQ9VFJVRSkgCmhlYWQoc2FtcGxlLmRlc2NyaXB0aW9uKQoKIyMgZ2V0IGFkZGl0aW9uYWwgbWV0YWRhdGEKc2FtcGxlLmluZm8gPC0gcmVhZHhsOjpyZWFkX2V4Y2VsKCIuLi9pbnB1dC93eTAwMV9STkFzZXEueGxzeCIsc2hlZXQgPSAxKQpoZWFkKHNhbXBsZS5pbmZvKQoKIyNjb21iaW5lCnNhbXBsZS5kZXNjcmlwdGlvbiA8LSBsZWZ0X2pvaW4oc2FtcGxlLmRlc2NyaXB0aW9uLCBzYW1wbGUuaW5mbykKc2FtcGxlLmRlc2NyaXB0aW9uIDwtIHNhbXBsZS5kZXNjcmlwdGlvbiAlPiUgCiAgbXV0YXRlKGdyb3VwPXBhc3RlKHRpc3N1ZSxnZW5vdHlwZSxzb2lsLGF1dG9jbGF2ZSxzZXA9Il8iKSkKaGVhZChzYW1wbGUuZGVzY3JpcHRpb24pCnNhbXBsZS5kZXNjcmlwdGlvbiAlPiUgc3VtbWFyaXplKG5fZGlzdGluY3QoZ3JvdXApKQpgYGAKCiMjIHN1bW1hcml6ZSBjb3VudHMKYGBge3J9CnBsLm9yaWcgPC0gY291bnRzWywtMV0gJT4lIGNvbFN1bXMoKSAlPiUgdGliYmxlKHNhbXBsZT1uYW1lcyguKSxjb3VudD0uKSAlPiUKICBnZ3Bsb3QoYWVzKHg9c2FtcGxlLHk9Y291bnQpKSArIAogIGdlb21fY29sKCkgKwogIHRoZW1lKGF4aXMudGV4dC54ICA9IGVsZW1lbnRfdGV4dChhbmdsZT05MCwgdmp1c3Q9MC41LHNpemUgPSA3KSkgCnBsLm9yaWcKYGBgCgojIyBsb2FkIHRvIGVkZ2VSLCBub3JtYWxpemUKCmBgYHtyfQojY29uZmlybSB0aGF0IGV2ZXJ0aGluZyBpcyBpbiB0aGUgcmlnaHQgb3JkZXIKYWxsKGNvbG5hbWVzKGNvdW50cylbLTFdPT1zYW1wbGUuZGVzY3JpcHRpb24kc2FtcGxlKQpkZ2UgPC0gREdFTGlzdChjb3VudHNbLC0xXSwKICAgICAgICAgICAgICAgZ3JvdXA9c2FtcGxlLmRlc2NyaXB0aW9uJGdyb3VwLAogICAgICAgICAgICAgICBzYW1wbGVzPXNhbXBsZS5kZXNjcmlwdGlvbiwKICAgICAgICAgICAgICAgZ2VuZXM9Y291bnRzJHRhcmdldF9pZCkKYGBgCgpgYGB7cn0KZGdlIDwtIGNhbGNOb3JtRmFjdG9ycyhkZ2UpCmJhcnBsb3QoZGdlJHNhbXBsZXMkbGliLnNpemUpCmdncGxvdChkZ2Ukc2FtcGxlcyxhZXMoeD1zYW1wbGUseT1ub3JtLmZhY3RvcnMsZmlsbD10aXNzdWUpKSArIGdlb21fY29sKCkgKyAKICB0aGVtZShheGlzLnRleHQueCAgPSBlbGVtZW50X3RleHQoYW5nbGU9OTAsIHZqdXN0PTAuNSxzaXplID0gNykpIApnZ3Bsb3QoZGdlJHNhbXBsZXMsYWVzKHg9c2FtcGxlLHk9bm9ybS5mYWN0b3JzLGZpbGw9Z2Vub3R5cGUpKSArIGdlb21fY29sKCkgKyAKICB0aGVtZShheGlzLnRleHQueCAgPSBlbGVtZW50X3RleHQoYW5nbGU9OTAsIHZqdXN0PTAuNSxzaXplID0gNykpIApnZ3Bsb3QoZGdlJHNhbXBsZXMsYWVzKHg9c2FtcGxlLHk9bm9ybS5mYWN0b3JzLGZpbGw9YXMuZmFjdG9yKGJsb2NrKSkpICsgZ2VvbV9jb2woKSArCiAgdGhlbWUoYXhpcy50ZXh0LnggID0gZWxlbWVudF90ZXh0KGFuZ2xlPTkwLCB2anVzdD0wLjUsc2l6ZSA9IDcpKSAKYGBgCgpMb29rcyBsaWtlIHdlIHNob3VsZCBub3JtYWxpemUgc2VwYXJhdGVseSBmb3Igcm9vdCBhbmQgbGVhZgoKIyBkbyBzZXBhcmF0ZWx5IGZvciBsZWFmIGFuZCByb290IHZhbHVlcwoKYGBge3J9CmNvdW50cy5sZWFmIDwtIGNvdW50cyAlPiUgc2VsZWN0KHRhcmdldF9pZCwgY29udGFpbnMoImxlYWYiKSkKY291bnRzLnJvb3QgPC0gY291bnRzICU+JSBzZWxlY3QodGFyZ2V0X2lkLCBjb250YWlucygicm9vdCIpKQpzYW1wbGUuZGVzY3JpcHRpb24ubGVhZiA8LSBzYW1wbGUuZGVzY3JpcHRpb24gJT4lIGZpbHRlcih0aXNzdWU9PSJsZWFmIikKc2FtcGxlLmRlc2NyaXB0aW9uLnJvb3QgPC0gc2FtcGxlLmRlc2NyaXB0aW9uICU+JSBmaWx0ZXIodGlzc3VlPT0icm9vdCIpCmBgYAoKTGVhZgpgYGB7cn0KI2NvbmZpcm0gdGhhdCBldmVydGhpbmcgaXMgaW4gdGhlIHJpZ2h0IG9yZGVyCmFsbChjb2xuYW1lcyhjb3VudHMubGVhZilbLTFdPT1zYW1wbGUuZGVzY3JpcHRpb24ubGVhZiRzYW1wbGUpCmRnZS5sZWFmIDwtIERHRUxpc3QoY291bnRzLmxlYWZbLC0xXSwKICAgICAgICAgICAgICAgICAgICBncm91cD1zYW1wbGUuZGVzY3JpcHRpb24ubGVhZiRncm91cCwKICAgICAgICAgICAgICAgICAgICBzYW1wbGVzPXNhbXBsZS5kZXNjcmlwdGlvbi5sZWFmLAogICAgICAgICAgICAgICAgICAgIGdlbmVzPWNvdW50cy5sZWFmJHRhcmdldF9pZCkKZGdlLmxlYWYgPC0gY2FsY05vcm1GYWN0b3JzKGRnZS5sZWFmKQpgYGAKClJvb3QKYGBge3J9CiNjb25maXJtIHRoYXQgZXZlcnRoaW5nIGlzIGluIHRoZSByaWdodCBvcmRlcgphbGwoY29sbmFtZXMoY291bnRzLnJvb3QpWy0xXT09c2FtcGxlLmRlc2NyaXB0aW9uLnJvb3Qkc2FtcGxlKQpkZ2Uucm9vdCA8LSBER0VMaXN0KGNvdW50cy5yb290WywtMV0sCiAgICAgICAgICAgICAgICAgICAgZ3JvdXA9c2FtcGxlLmRlc2NyaXB0aW9uLnJvb3QkZ3JvdXAsCiAgICAgICAgICAgICAgICAgICAgc2FtcGxlcz1zYW1wbGUuZGVzY3JpcHRpb24ucm9vdCwKICAgICAgICAgICAgICAgICAgICBnZW5lcz1jb3VudHMucm9vdCR0YXJnZXRfaWQpCmRnZS5yb290IDwtIGNhbGNOb3JtRmFjdG9ycyhkZ2Uucm9vdCkKYGBgCgpgYGB7cn0Kc2F2ZShkZ2UubGVhZixkZ2Uucm9vdCxzYW1wbGUuZGVzY3JpcHRpb24ubGVhZixzYW1wbGUuZGVzY3JpcHRpb24ucm9vdCxmaWxlPSIuLi9vdXRwdXQvZWRnZVJfZGdlX29iamVjdHMuUmRhdGEiKQpgYGAKCgojIyBXcml0ZSBvdXQgY3BtIHZhbHVlcwoKYGBge3J9CmNwbS5sZWFmLncgPC0gYmluZF9jb2xzKGRnZS5sZWFmJGdlbmUsYXNfdGliYmxlKGNwbShkZ2UubGVhZikpKSAlPiUgYXNfdGliYmxlKCkgJT4lIHJlbmFtZSh0cmFuc2NyaXB0X0lEPWdlbmVzKQpoZWFkKGNwbS5sZWFmLncpCndyaXRlX2NzdihjcG0ubGVhZi53LCIuLi9vdXRwdXQvY3BtX3dpZGVfMjAxNzA2MTdfbGVhZl9zYW1wbGVzLmNzdi5neiIpCmBgYAoKYGBge3J9CmNwbS5yb290LncgPC0gYmluZF9jb2xzKGRnZS5yb290JGdlbmUsYXNfdGliYmxlKGNwbShkZ2Uucm9vdCkpKSAlPiUgYXNfdGliYmxlKCkgJT4lIHJlbmFtZSh0cmFuc2NyaXB0X0lEPWdlbmVzKQpoZWFkKGNwbS5yb290LncpCndyaXRlX2NzdihjcG0ucm9vdC53LCIuLi9vdXRwdXQvY3BtX3dpZGVfMjAxNzA2MTdfcm9vdF9zYW1wbGVzLmNzdi5neiIpCmBgYAoKCkFsc28gbGV0J3MgcmVmb3JtYXQgdGhpcyB0byBsb25nIGZvcm1hdCBhbmQgYWRkIG1ldGFkYXRhCgpgYGB7cn0KY3BtLmxlYWYubG9uZyA8LSBjcG0ubGVhZi53ICU+JSAKICBnYXRoZXIoLXRyYW5zY3JpcHRfSUQsa2V5ID0gc2FtcGxlLHZhbHVlPWNwbSkgJT4lCiAgbGVmdF9qb2luKHNhbXBsZS5kZXNjcmlwdGlvbi5sZWFmKQpoZWFkKGNwbS5sZWFmLmxvbmcpCndyaXRlX2NzdihjcG0ubGVhZi5sb25nLCIuLi9vdXRwdXQvY3BtX2xvbmdfd2l0aF9tZXRhZGF0YV8yMDE3MDYxN19sZWFmX3NhbXBsZXMuY3N2Lmd6IikKYGBgCgpgYGB7cn0KY3BtLnJvb3QubG9uZyA8LSBjcG0ucm9vdC53ICU+JSAKICBnYXRoZXIoLXRyYW5zY3JpcHRfSUQsa2V5ID0gc2FtcGxlLHZhbHVlPWNwbSkgJT4lCiAgbGVmdF9qb2luKHNhbXBsZS5kZXNjcmlwdGlvbi5yb290KQpoZWFkKGNwbS5yb290LmxvbmcpCndyaXRlX2NzdihjcG0ucm9vdC5sb25nLCIuLi9vdXRwdXQvY3BtX2xvbmdfd2l0aF9tZXRhZGF0YV8yMDE3MDYxN19yb290X3NhbXBsZXMuY3N2Lmd6IikKYGBgCgoKIyMgY29tcHV0ZSBhbmQgd3JpdGUgb3V0IHZvb20tdHJhbnNmb3JtZWQgdmFsdWVzCgpgYGB7cn0KZGVzaWduLmxlYWYgPC0gbW9kZWwubWF0cml4KH4gc2FtcGxlLmRlc2NyaXB0aW9uLmxlYWYkZ3JvdXApCmRnZTR2b29tLmxlYWYgPC0gZGdlLmxlYWZbcm93U3VtcyhjcG0oZGdlLmxlYWYpPjEpID49IDYsLGtlZXAubGliLnNpemVzPUZBTFNFXQpkZ2U0dm9vbS5sZWFmIDwtIGNhbGNOb3JtRmFjdG9ycyhkZ2U0dm9vbS5sZWFmKQpkYXRhLnZvb20ubGVhZiA8LSB2b29tKGRnZTR2b29tLmxlYWYsZGVzaWduID0gZGVzaWduLmxlYWYpCmRhdGEudm9vbS5leHAubGVhZiA8LSBiaW5kX2NvbHMoZGF0YS52b29tLmxlYWYkZ2VuZXMsYXNfdGliYmxlKGRhdGEudm9vbS5sZWFmJEUpKSAlPiUKICByZW5hbWUodHJhbnNjcmlwdF9JRD1nZW5lcykgJT4lIGFzX3RpYmJsZSgpCmhlYWQoZGF0YS52b29tLmV4cC5sZWFmKQp3cml0ZV9jc3YoZGF0YS52b29tLmV4cC5sZWFmLCAiLi4vb3V0cHV0L3Zvb21fZXhwcmVzc2lvbl8yMDE3MDYxN19UNl9sZWFmX3NhbXBsZXMuY3N2Lmd6IikKYGBgCgoKYGBge3J9CnZvb20ubG9uZy5sZWFmIDwtIGRhdGEudm9vbS5leHAubGVhZiAlPiUgCiAgZ2F0aGVyKC10cmFuc2NyaXB0X0lELGtleSA9IHNhbXBsZSx2YWx1ZT1leHByZXNzaW9uKSAlPiUKICBsZWZ0X2pvaW4oc2FtcGxlLmRlc2NyaXB0aW9uLmxlYWYpCmhlYWQodm9vbS5sb25nLmxlYWYpCmhpc3Qodm9vbS5sb25nLmxlYWYkZXhwcmVzc2lvbikKd3JpdGVfY3N2KHZvb20ubG9uZy5sZWFmLCIuLi9vdXRwdXQvdm9vbV9sb25nX3dpdGhfbWV0YWRhdGFfMjAxNzA2MTdfVDZfbGVhZl9zYW1wbGVzLmNzdi5neiIpCmBgYAoKCmBgYHtyfQpkZXNpZ24ucm9vdCA8LSBtb2RlbC5tYXRyaXgofiBzYW1wbGUuZGVzY3JpcHRpb24ucm9vdCRncm91cCkKZGdlNHZvb20ucm9vdCA8LSBkZ2Uucm9vdFtyb3dTdW1zKGNwbShkZ2Uucm9vdCk+MSkgPj0gNiwsa2VlcC5saWIuc2l6ZXM9RkFMU0VdCmRnZTR2b29tLnJvb3QgPC0gY2FsY05vcm1GYWN0b3JzKGRnZTR2b29tLnJvb3QpCmRhdGEudm9vbS5yb290IDwtIHZvb20oZGdlNHZvb20ucm9vdCxkZXNpZ24gPSBkZXNpZ24ucm9vdCkKZGF0YS52b29tLmV4cC5yb290IDwtIGJpbmRfY29scyhkYXRhLnZvb20ucm9vdCRnZW5lcyxhc190aWJibGUoZGF0YS52b29tLnJvb3QkRSkpICU+JQogIHJlbmFtZSh0cmFuc2NyaXB0X0lEPWdlbmVzKSAlPiUgYXNfdGliYmxlKCkKaGVhZChkYXRhLnZvb20uZXhwLnJvb3QpCndyaXRlX2NzdihkYXRhLnZvb20uZXhwLnJvb3QsICIuLi9vdXRwdXQvdm9vbV9leHByZXNzaW9uXzIwMTcwNjE3X1Q2X3Jvb3Rfc2FtcGxlcy5jc3YuZ3oiKQpgYGAKCgpgYGB7cn0Kdm9vbS5sb25nLnJvb3QgPC0gZGF0YS52b29tLmV4cC5yb290ICU+JSAKICBnYXRoZXIoLXRyYW5zY3JpcHRfSUQsa2V5ID0gc2FtcGxlLHZhbHVlPWV4cHJlc3Npb24pICU+JQogIGxlZnRfam9pbihzYW1wbGUuZGVzY3JpcHRpb24ucm9vdCkKaGVhZCh2b29tLmxvbmcucm9vdCkKaGlzdCh2b29tLmxvbmcucm9vdCRleHByZXNzaW9uKQp3cml0ZV9jc3Yodm9vbS5sb25nLnJvb3QsIi4uL291dHB1dC92b29tX2xvbmdfd2l0aF9tZXRhZGF0YV8yMDE3MDYxN19UNl9yb290X3NhbXBsZXMuY3N2Lmd6IikKYGBgCgp3cml0ZSBpdCB0byBpcm9kcwoKTmVlZCB0byBydW4gdGhpcyB5b3Vyc2VsZiBpbiB0ZXJtaW5hbAoKYGBge3IsIGVuZ2luZT0nYmFzaCcsIGV2YWw9RkFMU0V9Cmlpbml0CmljZCAvaXBsYW50L2hvbWUvc2hhcmVkL3VjZC5icmFzc2ljYS9hbmFseXNlcy9CcmFwYV9NaWNyb2Jpb21lX1JOQXNlcS8KZm9yIGYgaW4gKGxzIGNwbSopCiAgICBlY2hvICRmCiAgICBpcHV0IC12ZiAkZgplbmQKZm9yIGYgaW4gKGxzIHZvb20qKQogICAgZWNobyAkZgogICAgaXB1dCAtdmYgJGYKZW5kCmBgYAoKIyMgcmVhZCBkaXN0cmlidXRpb24KCk1pa2UgYXNrZWQgaWYgdGhlIGRpZmZlcmVuY2UgaW4gbm9ybWFsaXphdGlvbiBmYWN0b3JzIGluIGxlYWZzIHZzIHJvb3RzIHdhcyBkdWUgdG8gaGlnaCBhYnVuZGFuY2Ugb2YgcGhvdG9zeW50aGVzaXMgdHJhbnNjcmlwdHMgaW4gbGVhZnMuICAoQWx0aG91Z2ggbGVhZnMgaGF2ZSBsb3dlciBub3JtYWxpemF0aW9uIGZhY3RvcikKClN0YXJ0IGJ5IGRvaW5nIGEgc2ltcGxpc3RpYyBub3JtYWxpemF0aW9uIGp1c3QgYnkgbGlicmFyeSBzaXplLiAgVGhlbiBsb29rIGF0IGRpc3RyaWJ1dGlvbiBvZiBtb3N0IGFidW5kYW50IGNvdW50cy4KCmBgYHtyfQpjb3VudHMubGVhZi5ub3JtIDwtIGNvdW50cy5sZWFmICU+JSAKICBtdXRhdGVfYXQoLTEsZnVucyguL3N1bSguKSkpICU+JSAKICB0cmFuc211dGUodGFyZ2V0X2lkPXRhcmdldF9pZCxtZWFuPSB7c2VsZWN0KC4sLXRhcmdldF9pZCkgJT4lIHJvd01lYW5zKCl9KSAlPiUKICBhcnJhbmdlKGRlc2MobWVhbikpICU+JSBtdXRhdGUoY3Vtc3VtPWN1bXN1bShtZWFuKSxyYW5rPXJvd19udW1iZXIoKSxzYW1wbGU9ImxlYWYiKQpjb3VudHMubGVhZi5ub3JtCmBgYApgYGB7cn0KY291bnRzLnJvb3Qubm9ybSA8LSBjb3VudHMucm9vdCAlPiUgCiAgbXV0YXRlX2F0KC0xLGZ1bnMoLi9zdW0oLikpKSAlPiUgCiAgdHJhbnNtdXRlKHRhcmdldF9pZD10YXJnZXRfaWQsbWVhbj0ge3NlbGVjdCguLC10YXJnZXRfaWQpICU+JSByb3dNZWFucygpfSkgJT4lCiAgYXJyYW5nZShkZXNjKG1lYW4pKSAlPiUgbXV0YXRlKGN1bXN1bT1jdW1zdW0obWVhbikscmFuaz1yb3dfbnVtYmVyKCksc2FtcGxlPSJyb290IikKY291bnRzLnJvb3Qubm9ybQpgYGAKCmBgYHtyfQpyYmluZChjb3VudHMubGVhZi5ub3JtLGNvdW50cy5yb290Lm5vcm0pICU+JQogIGdncGxvdChhZXMoeD1yYW5rLHk9Y3Vtc3VtLGNvbG9yPXNhbXBsZSkpICsKICBnZW9tX2xpbmUoKSAreGxpbSgwLDIwMDAwKQpgYGAKCmBgYHtyfQpyYmluZChjb3VudHMubGVhZi5ub3JtLGNvdW50cy5yb290Lm5vcm0pICU+JQogIGdncGxvdChhZXMoeD1yYW5rLHk9bWVhbixjb2xvcj1zYW1wbGUpKSArCiAgZ2VvbV9saW5lKCkgICsgc2NhbGVfeV9sb2cxMCgpCmBgYAoKYGBge3J9CnJiaW5kKGNvdW50cy5sZWFmLm5vcm0sY291bnRzLnJvb3Qubm9ybSkgJT4lIGZpbHRlcihyYW5rIDwgNDEpICU+JQogIGdncGxvdChhZXMoeD1yYW5rLHk9bWVhbixmaWxsPXNhbXBsZSkpICsKICBnZW9tX2NvbChwb3NpdGlvbiA9ICJkb2RnZSIpIApgYGAKCmBgYHtyfQphbm5vdGF0aW9uIDwtIHJlYWRfY3N2KCIuLi8uLi8uLi9Bbm5vdGF0aW9uL291dHB1dC92My4wYW5ub3RhdGlvbi9CcmFwYV9WMy4wX2Fubm90YXRlZC5jc3YiKQpgYGAKCmBgYHtyfQp0b3AuZXhwcmVzc2VkLmxlYWYgPC0gY291bnRzLmxlYWYubm9ybSAlPiUgZmlsdGVyKHJhbms8MjEpICU+JSBsZWZ0X2pvaW4oYW5ub3RhdGlvbixieT1jKCJ0YXJnZXRfaWQiPSJuYW1lIikpICU+JSBzZWxlY3QoInRhcmdldF9pZCIsIm1lYW4iLCJyYW5rIiwiQUdJIiwiQXRfc3ltYm9sIiwiQXRfZGVzY3JpcHRpb24iKSAlPiUgYXJyYW5nZShyYW5rKQp0b3AuZXhwcmVzc2VkLmxlYWYKd3JpdGVfY3N2KHRvcC5leHByZXNzZWQubGVhZiwiLi4vb3V0cHV0L3RvcC5leHByZXNzZWQubGVhZi5jc3YiKQpgYGAKCmBgYHtyfQp0b3AuZXhwcmVzc2VkLnJvb3QgPC0gY291bnRzLnJvb3Qubm9ybSAlPiUgZmlsdGVyKHJhbms8MjEpICU+JSBsZWZ0X2pvaW4oYW5ub3RhdGlvbixieT1jKCJ0YXJnZXRfaWQiPSJuYW1lIikpICU+JSBzZWxlY3QoInRhcmdldF9pZCIsIm1lYW4iLCJyYW5rIiwiQUdJIiwiQXRfc3ltYm9sIiwiQXRfZGVzY3JpcHRpb24iKSAlPiUgYXJyYW5nZShyYW5rKQp0b3AuZXhwcmVzc2VkLnJvb3QKd3JpdGVfY3N2KHRvcC5leHByZXNzZWQucm9vdCwiLi4vb3V0cHV0L3RvcC5leHByZXNzZWQucm9vdC5jc3YiKQpgYGA=